 @include "lib.awk" # standard utils

 BEGIN { FS="," ; FindClass="theClass"}

 function readcsv(name,lo,hi,goals,d,w,has,csv,logp,  r) {
   csv = csv ? csv : "cat -"  
   while((csv | getline ) > 0) {   # for every line do
     gsub(/[ \t\r]/,"")            # kill whitespace
     gsub(/(#.*|[ \t'"])/,"")      # kill comments
     if (/^$/) continue            # skip empty lines
     if (r) # then the header is read and we can process the data
	 readline(name,lo,hi,goals,w,has,d,r,logp) 
     else # we have seen no lines yet; time to read the header
	 readhead(name,lo,hi,goals,w) 
     r++  }                                                 
   close(csv)
 }
 function readhead(name,lo,hi,goals,w,   c) {
   for(c=1;c<=NF;c++) {
     if ($c ~ /\?/) continue # skip the stuff we are ignoring                   
     w[c] = weight($c)       # record feature weights
     if ($c ~ /!/) goals[c]  # record if this is a goal        
     if ($c ~ /\$/)          # if this a numeric feature, then...
       hi[c]= -1 * (lo[c] = 10^10) # ... initialize the hi and lo values
     gsub(/\*.*/,"",$c)      # wipe the weight info
     name[c] = $c            # record the name             
     
 }}
 function weight(colname,   n,tmp) {
   n = split(colname,tmp,/\*/)    # does "name" look like "name*n" ?
   return n==1 ? 1 : tmp[2]*1     # if not, weight = 1. else, weight = n
 }
 function readline(name,lo,hi,goals,w,has,d,r,logp,   \
                   val,class,c) {
   class = @FindClass(goals) # summarize goals to one "class" value
   for(c in w) { 
     val = $c
     if (logp && c in lo && val != "?")  # maybe log numerics
       val = log(val < 0.0001 ? 0.0001 : val)
     d[r][c] = val      # store data
     if (val !~ /?/) {  # if we aren't ignoring this cell...
       has[c][val][class]++   
       if (c in lo) {   # update any info on numerics
         if (val > hi[c]) hi[c] = val
         if (val < lo[c]) lo[c] = val }}}
 }
 function theClass(goals,r,d,    c) { 
   if (r) for(c in goals) return d[r][c] # for data from the "d" array
   else   for(c in goals) return $c      # for data from standard input
 }
